package com.wdl.modules.myhouse.spider;

import com.wdl.modules.myhouse.vo.spider.FileSpider;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.annotations.Select;
import org.jsoup.Connection;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.concurrent.DefaultManagedAwareThreadFactory;
import org.springframework.stereotype.Component;
import reactor.util.annotation.Nullable;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Array;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.rmi.RemoteException;
import java.util.*;
import java.util.concurrent.*;

/**
 * @author 昏昏的君
 */
@Slf4j
@Component
public class CategoryFileSpider {

    @Autowired
    ThreadPoolExecutor pool;

    public static void main(String[] args) throws IOException {
        Set<String> strings = allClassify();
        System.out.println(
                strings
        );
    }

    public List<FileSpider> spiderTopicRes() throws IOException {
        //横着的 "修改软件","软件合集"
//        String[] rootLinks = new String[]{
//                "https://www.ghxi.com/special/ossoft",
//                "https://www.ghxi.com/special/ghlr",
//        };
        //竖着的
        String[] rootLinks = new String[]{
                "https://www.ghxi.com/special/softlist",
                "https://www.ghxi.com/special/jqsoft",
                "https://www.ghxi.com/special/help",
        };
        String[] catInfos = new String[]{

                "软件合集",
                "惊奇软件",
                "帮助文档"

        };
        ArrayList<Future<?>> futures = new ArrayList<>();
        CompletableFuture<List<FileSpider>> future = null;
        ArrayList<FileSpider> fileSpiders = new ArrayList<>();
        int i = 0;
        for (String rootLink : rootLinks) {
            int pageNum = urlPageNumMapping(rootLink);
            String catInfo = catInfos[i++];
            List<String> pageLinks = buildListLinkSet(rootLink, pageNum);
            if (pageLinks != null) {
                getTopicData(catInfo, rootLink, pageLinks, fileSpiders);
            }
        }
        return fileSpiders;
    }

    /**
     * 爬取主题资源代码
     */
    public List<FileSpider> getTopicData(String catInfo, String catLinks, List<String> pageLinks, List<FileSpider> spiderFiles) {
        log.info(catLinks + "开始爬取");
        log.info("============================");
        for (String pageLink : pageLinks) {
            Document pageDoc = null;
            try {
                pageDoc = Jsoup.parse(new URL(pageLink).openStream(), "UTF-8", pageLink);
            } catch (IOException e) {
                e.printStackTrace();
            }
            //                    Document pageDoc = Jsoup.connect(pageLink).get();
            List<String> currentPageLinks = null;
            List<String> imgLinks = null;
            try {
                currentPageLinks = getListLink(pageLink, pageDoc);
                imgLinks = getImgLinks(pageLink, pageDoc);
            } catch (IOException e) {
                e.printStackTrace();
            }

            // 文章链接
            int i = 0;
            log.info(pageLink + ":开始爬取");
            log.info("------------------------------");
            for (String link : currentPageLinks) {
                FileSpider fileSpider = new FileSpider();
                //教育教学,
                Set<String> downloadLinks = null;
                String[] titleAndContent = new String[0];
                try {
                    Document document = Jsoup.parse(new URL(link).openStream(), "UTF-8", link);
                    //                        Document document = Jsoup.connect(link).get();
                    downloadLinks = getDownloadLinks(link, document);
                    titleAndContent = getContent(link, document);
                } catch (IOException e) {
                    e.printStackTrace();
                }


                fileSpider.setArticle(titleAndContent[1]);
                fileSpider.setArticleTitle(titleAndContent[0]);
                fileSpider.setDownloadLinks(downloadLinks);
                fileSpider.setFileName(titleAndContent[0]);
                fileSpider.setFileTypeName(catInfo);
                fileSpider.setTitleImgUrl(i < imgLinks.size() ? imgLinks.get(i) : "");
                fileSpider.setSourceLink(pageLink);
                i++;
                spiderFiles.add(fileSpider);
            }
        }
        return spiderFiles;
    }


    /**
     * 多线程版本
     *
     * @throws IOException
     */
    public List<FileSpider> runAsync() throws IOException {

        Set<String> rootLinks = allClassify();
//        Set<String> rootLinks= new HashSet<String>();
//        rootLinks.addAll(Arrays.asList("https://www.ghxi.com/category/all/pcsoft/pctools/sysplus","https://www.ghxi.com/category/all/pcsoft/pctools/sysplus"));
        //所有分类url： https://www.ghxi.com/category/all/system/win11
        ArrayList<Future<?>> futures = new ArrayList<>();
        CompletableFuture<List<FileSpider>> future = null;
        ArrayList<CompletableFuture<List<FileSpider>>> spiderFuture = new ArrayList<>();
        for (String catLinks : rootLinks) {
            int pageNum = urlPageNumMapping(catLinks);
            //根据url获得分类名
            String catInfo = getCatInfo(catLinks);
            //添加分页信息 [https://www.ghxi.com/category/all/system/win11/page/1,https://www.ghxi.com/category/all/system/win11/]
            List<String> pageLinks = buildListLinkSet(catLinks, pageNum);
            if (pageLinks != null) {
                //当前页面所有资源url  (pageLinks 所有分页资源)
                future = CompletableFuture.supplyAsync(() -> {
                    return getCatData(catInfo, catLinks, pageLinks);
                }, pool);
                spiderFuture.add(future);
            }
        }
        CompletableFuture.allOf(spiderFuture.toArray(new CompletableFuture[]{}));
        ArrayList<FileSpider> fileSpiders = new ArrayList<>();
        for (CompletableFuture<List<FileSpider>> future1 : spiderFuture) {
            try {
                List<FileSpider> fileSpiders1 = future1.get();
                merger(fileSpiders, fileSpiders1);
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        return fileSpiders;
    }

    private <T> void merger(List<T> target, List<T> source) {
        for (T item : source) {
            target.add(item);
        }
    }

    /**
     * 第二个多线程版本
     */
    @Deprecated
    public void runAsync2(List<FileSpider> spiderFiles) throws IOException, InterruptedException {

        Set<String> rootLinks = allClassify();
        //所有分类url： https://www.ghxi.com/category/all/system/win11
        ArrayList<Future<?>> futures = new ArrayList<>();
        Thread thread = null;
        ArrayList<Thread> threads = new ArrayList<>();
        for (String catLinks : rootLinks) {
            int pageNum = urlPageNumMapping(catLinks);
            //根据url获得分类名
            String catInfo = getCatInfo(catLinks);


            //添加分页信息 [https://www.ghxi.com/category/all/system/win11/page/1,https://www.ghxi.com/category/all/system/win11/]
            List<String> pageLinks = buildListLinkSet(catLinks, pageNum);
            if (pageLinks != null) {
                //当前页面所有资源url  (pageLinks 所有分页资源)
                thread = new Thread(() -> {
                    getCatData(catInfo, catLinks, pageLinks);
                });
                threads.add(thread);
            }
        }
        for (Thread thread1 : threads) {
            thread1.start();
            thread1.join();
        }

    }


    /**
     * @param catInfo     分类名
     * @param catLinks    分类路径
     * @param pageLinks   分页路径信息
     * @param spiderFiles 总体文件信息
     * @throws IOException
     */
    public List<FileSpider> getCatData(String catInfo, String catLinks, List<String> pageLinks) {
        log.info(catLinks + "开始爬取");
        log.info("============================");
        List<FileSpider> spiderFiles = new ArrayList<>();
        for (String pageLink : pageLinks) {
            Document pageDoc = null;
            try {
                pageDoc = Jsoup.parse(new URL(pageLink).openStream(), "UTF-8", pageLink);
            } catch (IOException e) {
                e.printStackTrace();
            }
            //                    Document pageDoc = Jsoup.connect(pageLink).get();
            List<String> currentPageLinks = null;
            List<String> imgLinks = null;
            try {
                currentPageLinks = getListLink(pageLink, pageDoc);
                imgLinks = getImgLinks(pageLink, pageDoc);
            } catch (IOException e) {
                e.printStackTrace();
            }

            // 文章链接
            int i = 0;
            log.info(pageLink + ":开始爬取");
            log.info("------------------------------");
            for (String link : currentPageLinks) {
                FileSpider fileSpider = new FileSpider();
                //教育教学,
                Set<String> downloadLinks = null;
                String[] titleAndContent = new String[0];
                try {
                    Document document = Jsoup.parse(new URL(link).openStream(), "UTF-8", link);
                    //                        Document document = Jsoup.connect(link).get();
                    downloadLinks = getDownloadLinks(link, document);
                    titleAndContent = getContent(link, document);
                } catch (IOException e) {
                    e.printStackTrace();
                }


                fileSpider.setArticle(titleAndContent[1]);
                fileSpider.setArticleTitle(titleAndContent[0]);
                fileSpider.setDownloadLinks(downloadLinks);
                fileSpider.setFileName(titleAndContent[0]);
                fileSpider.setFileTypeName(catInfo);
                fileSpider.setTitleImgUrl(i < imgLinks.size() ? imgLinks.get(i) : "");
                fileSpider.setSourceLink(pageLink);
                i++;
                spiderFiles.add(fileSpider);
            }
        }
        return spiderFiles;
    }


    public List<FileSpider> run() throws IOException {
        ArrayList<FileSpider> spiderFiles = new ArrayList<>();
        Set<String> rootLinks = allClassify();
        //所有分类url： https://www.ghxi.com/category/all/system/win11
        try {
            for (String catLinks : rootLinks) {
                int pageNum = urlPageNumMapping(catLinks);
                //根据url获得分类名
                String catInfo = getCatInfo(catLinks);

                //添加分页信息 [https://www.ghxi.com/category/all/system/win11/page/1,https://www.ghxi.com/category/all/system/win11/]
                List<String> pageLinks = buildListLinkSet(catLinks, pageNum);
                if (pageLinks != null) {
                    //当前页面所有资源url  (pageLinks 所有分页资源)
                    log.info(catLinks + "开始爬取");
                    log.info("============================");
                    for (String pageLink : pageLinks) {
                        Document pageDoc = Jsoup.parse(new URL(pageLink).openStream(), "UTF-8", pageLink);
                        //                    Document pageDoc = Jsoup.connect(pageLink).get();
                        List<String> currentPageLinks = getListLink(pageLink, pageDoc);
                        List<String> imgLinks = getImgLinks(pageLink, pageDoc);
                        // 文章链接
                        int i = 0;
                        log.info(pageLink + ":开始爬取");
                        log.info("------------------------------");
                        for (String link : currentPageLinks) {
                            log.info("开始爬取: " + link);
                            FileSpider fileSpider = new FileSpider();
                            //教育教学,
                            Document document = Jsoup.parse(new URL(link).openStream(), "UTF-8", link);
                            //                        Document document = Jsoup.connect(link).get();
                            Set<String> downloadLinks = getDownloadLinks(link, document);
                            String[] titleAndContent = getContent(link, document);


                            fileSpider.setArticle(titleAndContent[1]);
                            fileSpider.setArticleTitle(titleAndContent[0]);
                            fileSpider.setDownloadLinks(downloadLinks);
                            fileSpider.setFileName(titleAndContent[0]);
                            fileSpider.setFileTypeName(catInfo);
                            fileSpider.setTitleImgUrl(i < imgLinks.size() ? imgLinks.get(i) : "");
                            fileSpider.setSourceLink(pageLink);
                            i++;
                            spiderFiles.add(fileSpider);
                            log.info("爬取完毕: " + link);
                        }
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return spiderFiles;
    }

    /**
     * 根据路径获取分类信息
     */
    public static String getCatInfo(String path) {
        //https://www.ghxi.com/category/all/system/win11
        String rootAndChild = null;
        if (path.length() >= 34) {
            rootAndChild = path.substring(34);
            if (rootAndChild.length() > 0) {
                String root = rootAndChild.split("/")[0];
                String child = null;
                try {
                    child = rootAndChild.split("/")[1];
                } catch (ArrayIndexOutOfBoundsException e) {
                    log.info("根路径： " + path);
                }
                return child == null ? root : child;
            } else {
                throw new RuntimeException("未识别的类型" + path);
            }
        } else {
            if (path.contains("special")) {
                return "special";
            } else if (path.endsWith("all")) {
                return "all";
            } else {
                throw new RuntimeException("未识别的类型" + path);
            }
        }
    }


    /**
     * 爬取菜单分级信息
     */
    public static Map<String, Set<String>> getMenuTree() throws IOException {
//        if (url.contains("%")) {
//            url = URLDecoder.decode("https://www.ghxi.com/%e2%80%8b%e2%80%8bresumebuilder.html", "UTF-8");
//        }
        Document document = Jsoup.connect("https://www.ghxi.com/category/all/android/xposed").get();
        Elements elements = document.select(".primary-menu");
        Elements eles = elements.select(".menu-item");
        HashSet<String> set = new HashSet<>();
        for (Element ele : eles) {
            String attr = ele.select("a").attr("href");
            set.add(attr);
        }
        // 根路径--》【子路径，子路径】
        HashMap<String, Set<String>> mapper = new HashMap<>();
        mapper.put("all", null);
        mapper.put("special", null);
        //path:    https://www.ghxi.com/category/all/

        for (String path : set) {
            //pcsoft/ipcsoft
            System.out.println(path);
            String rootAndChild = null;
            if (path.length() >= 34) {
                rootAndChild = path.substring(34);
                if (rootAndChild.length() > 0) {
                    String root = rootAndChild.split("/")[0];
                    String child = null;
                    try {
                        child = rootAndChild.split("/")[1];
                    } catch (ArrayIndexOutOfBoundsException e) {
                        log.info("根路径： " + path);
                    }
                    //当前根节点对应的子节点集合
                    Set<String> currentRootChild = mapper.get(root);
                    if (currentRootChild == null) {
                        currentRootChild = new HashSet<>();
                    }
                    if (child != null) {
                        currentRootChild.add(child);
                    }
                    mapper.put(root, currentRootChild);
                }
            }
        }
        return mapper;
    }


    /**
     * 获取某一类的分页信息
     */
    public static int urlPageNumMapping(String url) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode("https://www.ghxi.com/%e2%80%8b%e2%80%8bresumebuilder.html", "UTF-8");
        }
        Document document = Jsoup.connect(url).get();
        String span = document.select(".pagination").select(".disabled").select("span").text();
        if (span.isEmpty()) {
            return 1;
        }
        String pageNum = span.replaceAll(" ", "").split("/")[1];
        return Integer.parseInt(pageNum);
    }


    /**
     * 获取所有分类url信息
     */
    public static Set<String> allClassify() throws IOException {
//        if (url.contains("%")) {
//            url = URLDecoder.decode("https://www.ghxi.com/%e2%80%8b%e2%80%8bresumebuilder.html", "UTF-8");
//        }
        Document document = Jsoup.connect("https://www.ghxi.com/category/all/android/xposed").get();
        Elements elements = document.select(".primary-menu");
        Elements eles = elements.select(".menu-item");
        HashSet<String> set = new HashSet<>();
        for (Element ele : eles) {
            String attr = ele.select("a").attr("href");
            set.add(attr);
        }
        return set;
    }


    /**
     * 构建资源链接集合
     *
     * @param url       基础链接比如： https://www.ghxi.com/category/all
     * @param root      一级分类 比如：/android
     * @param child     分类子菜单比如： /xposed
     * @param totalPage 总页码
     * @return 列表链接集合
     * @throws IOException
     */
    public static List<String> buildListLinkSet(String url, String root, String child, int totalPage) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        String firstPageUrl = url + root + child;
        if (totalPage <= 1) {
            return Arrays.asList(firstPageUrl);
        } else {
            ArrayList<String> links = new ArrayList<>();
            for (int i = 2; i <= totalPage; i++) {
                links.add(url + root + child + "/page/" + i);
            }
            return links;
        }
    }

    /**
     * 构建资源链接集合
     *
     * @param url       基础链接比如： https://www.ghxi.com/category/all
     * @param totalPage 总页码
     * @return 列表链接集合
     * @throws IOException
     */
    public static List<String> buildListLinkSet(String url, int totalPage) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        String firstPageUrl = url;
        if (totalPage <= 1) {
            return Arrays.asList(firstPageUrl);
        } else {
            ArrayList<String> links = new ArrayList<>();
            for (int i = 2; i <= totalPage; i++) {
                links.add(url + "/page/" + i);
            }
            return links;
        }
    }


    /**
     * 获取文章内容及其标题,title,content
     */
    public static String[] getContent(String url, Document document) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        if (document == null) {
            document = Jsoup.connect(url).get();
        }
        Elements mainContent = document.select(".entry-content");
        mainContent.select(".vip-center-vip-show").remove();
        mainContent.select(".entry-copyright").remove();
        mainContent.select(".vipdown-form").remove();
        mainContent.select("vipdown-form-loginnotetext").remove();
        String title = document.select(".entry-title").text();
        return new String[]{title, mainContent.toString()};
    }

    /**
     * 爬取下载链接或者下载地址
     *
     * @throws IOException
     */
    public static Set<String> getDownloadLinks(String url, Document document) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        if (document == null) {
            document = Jsoup.connect(url).get();
        }
        Elements elements = document.select("h2:contains(下载地址)");
        if (elements.size() == 0) {
            elements = document.select("h2:contains(下载链接)");
        }
        if (elements.size() == 0) {
            log.warn(url + "找不到下载地址");
            HashSet<String> set = new HashSet<>();
            set.add("找不到下载地址，或为圈内新闻");
            return set;
        }
        HashSet<String> set = new HashSet<>();
        StringBuilder sourceBuilder = new StringBuilder();
        int len = 0;
        do {
            sourceBuilder.append(elements);
            elements = elements.next();
            set.add(elements.select("a").attr("href"));
            len++;
        } while (!elements.hasClass("vipdown-form") && !elements.hasClass("entry-copyright") && len < 10);
        return set;

    }


    /**
     * 获取列表元素所指向的链接
     *
     * @throws IOException
     */
    public static List<String> getListLink(String url, Document document) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        if (document == null) {
            document = Jsoup.connect(url).get();
        }
        Elements eles = document.select(".item-img-inner");
//        Elements eles = document.select(".item-thumb");
        ArrayList<String> list = new ArrayList<>();
        for (Element ele : eles) {
            list.add(ele.attr("href"));
        }
        return list;
    }

    /**
     * 获取图像信息
     *
     * @return
     */
    public static List<String> getImgLinks(String url, Document document) throws IOException {
        if (url.contains("%")) {
            url = URLDecoder.decode(url, "UTF-8");
        }
        if (document == null) {
            document = Jsoup.connect(url).get();
        }
        Elements eles = document.select(".item-img-inner").select("img");
//        Elements eles = document.select(".attachment-post-thumbnail");
//        System.out.println(eles);
        ArrayList<String> list = new ArrayList<>();
        for (Element ele : eles) {
            list.add(ele.attr("data-original"));
        }
        return list;
    }

}
